{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Reinforcement Learning <em> in Action </em> \n",
    "## Ch. 4 - Policy Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import torch\n",
    "from matplotlib import pylab as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def running_mean(x, N=50):\n",
    "    cumsum = np.cumsum(np.insert(x, 0, 0)) \n",
    "    return (cumsum[N:] - cumsum[:-N]) / float(N)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Defining Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = 4\n",
    "l2 = 150\n",
    "l3 = 2\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.LeakyReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.Softmax()\n",
    ")\n",
    "\n",
    "learning_rate = 0.0009\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Objective Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(preds, r): \n",
    "    # pred is output from neural network, a is action index\n",
    "    # r is return (sum of rewards to end of episode), d is discount factor\n",
    "    return -torch.sum(r * torch.log(preds)) # element-wise multipliy, then sum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  input = module(input)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "MAX_DUR = 200\n",
    "MAX_EPISODES = 500\n",
    "gamma_ = 0.99\n",
    "time_steps = []\n",
    "for episode in range(MAX_EPISODES):\n",
    "    curr_state = env.reset()\n",
    "    done = False\n",
    "    transitions = [] # list of state, action, rewards\n",
    "    \n",
    "    for t in range(MAX_DUR): #while in episode\n",
    "        act_prob = model(torch.from_numpy(curr_state).float())\n",
    "        action = np.random.choice(np.array([0,1]), p=act_prob.data.numpy())\n",
    "        prev_state = curr_state\n",
    "        curr_state, reward, done, info = env.step(action)\n",
    "        transitions.append((prev_state, action, reward))\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "    # Optimize policy network with full episode\n",
    "    ep_len = len(transitions) # episode length\n",
    "    time_steps.append(ep_len)\n",
    "    preds = torch.zeros(ep_len)\n",
    "    discounted_rewards = torch.zeros(ep_len)\n",
    "    for i in range(ep_len): #for each step in episode\n",
    "        discount = 1\n",
    "        future_reward = 0\n",
    "        # discount rewards\n",
    "        for i2 in range(i, ep_len):\n",
    "            future_reward += transitions[i2][2] * discount\n",
    "            discount = discount * gamma_\n",
    "        discounted_rewards[i] = future_reward\n",
    "        state, action, _ = transitions[i]\n",
    "        pred = model(torch.from_numpy(state).float())\n",
    "        preds[i] = pred[action]\n",
    "    loss = loss_fn(preds, discounted_rewards)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11c57dba8>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmgAAAGtCAYAAACiIojTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd4VFXixvHvSQgEAoQQei/SFKSFjkgvogJiQ1RUEAssNlzLb1FwV0VZKyqgooIIiICgiEgNAoKSEEroLRAgJHRCgLQ5vz+IbNAAATJzJ5P38zx5mDn3zsw7+6zw5t5z7zHWWkRERETEe/g5HUBERERELqSCJiIiIuJlVNBEREREvIwKmoiIiIiXUUETERER8TIqaCIiIiJeRgVNRERExMuooImIiIh4GRU0ERERES+Tz+kA16JEiRK2SpUqTscQERERuazIyMjD1tqS2dk3Vxe0KlWqEBER4XQMERERkcsyxuzJ7r46xSkiIiLiZVTQRERERLyM2wqaMaaiMWaJMWazMWajMeapjPHixpgFxpjtGX+GZIwbY8yHxpgdxpj1xphG7somIiIi4s3ceQQtDXjOWlsHaA4MMsZcD7wILLLW1gAWZTwH6AbUyPgZCIxxYzYRERERr+W2gmatjbPWrsl4nAhsBsoDPYAJGbtNAHpmPO4BTLTnrAKKGWPKuiufiIiIiLfyyBw0Y0wVoCHwO1DaWhsH50ocUCpjt/JAbKaX7csYExEREclT3F7QjDGFgRnA09bak5faNYsxm8X7DTTGRBhjIg4dOpRTMUVERES8hlsLmjEmgHPl7Btr7cyM4fg/T11m/JmQMb4PqJjp5RWAA399T2vtp9baMGttWMmS2brXm4iIiEiu4s6rOA0wHthsrX0306YfgH4Zj/sBszONP5hxNWdz4MSfp0JFRERE8hJ3riTQCngA2GCMWZsx9jIwEphmjOkP7AXuytg2F7gF2AGcBh52YzYRERERr+W2gmatXU7W88oAOmSxvwUGuSuPiIiISG6hlQREREREvIwKmoiIiIiXUUETERER8TIqaCIiInlAUkoSqempTseQbFJBExER8XEu66L1l62pP7Y+e47vIc2V5nQkuQx33mZDREREvMCCnQtYe3AtBkOVD6pQPaQ6mwdtJsA/wOlochEqaCIiIj5k7va5PDL7EfL55aNN5TaEFgwlfE84ZQqX4cc+PzJh7QQ+Wv0Rv+z8hVtr3up0XLkIFTQREREfMnnDZFLSU2hXtR1L9yzlbNpZDIYRbUcQVi6M+qXrM23TNL5c+6UKmhdTQRMREfER1loW715M1+u6Mrn35Cz3CfAP4P569zP6j9H8uudXmldozqp9qwgrF0ahgEIeTiwXo4sEREREfMTWI1uJOxVH+6rtL7nfi61f5Lri19F+QnuCRwZz81c3M2zxMA+llOxQQRMREfEBy/Ys4/kFzwPQrkq7S+5bMqgkS/ot4fmWz/Nk2JO0q9KOz6M+JzE50RNRJRt0ilNERCQXiD0RS3xSPGHlwrLc/ticx9h2ZButK7WmWki1y75f6cKlebPjmwCs2reKFuNbMGHdBAY3HZyjueXqqKCJiIh4qYnrJrL18FZSXal8GvkpSalJLHxgITdXufmC/TYd2sTmw5v5+JaPebLJk1f8Oc0rNKdZ+WZ8+PuHPNnkSfyMTrA5TQVNRETEC+08upN+s/rhZ/zwM340Ld+UI6eP0Htab+bdP4+U9BTunX4v8UnxVCxaEYPhjjp3XPXnPdXsKe6beR9To6fSq3YvCgYUzMFvI1dKFVlERMQLfbX2KwyGPU/vIXVYKiseWcGc++ZQOH9hmn7WlNZftKZAvgL0q9+PXcd20aZyG8oULnPVn3fn9XdSrkg5+s7sS5UPqjBvx7wc/DZypYy11ukMVy0sLMxGREQ4HUNERCTHnE49zX9+/Q/jo8bTqGwjfu778wXb4xLjGBc5Dj/jxxNhT1AyqCTr49dTvGBxKhStcE2fHZ0Qzap9q/jw9w/ZfHgzyx9eTrMKza7pPeV/jDGR1tqsJxH+dV8VNBEREe8xZvUYnpz7JCULlWTSHZPoXL2zxzMcO3OMRp82wlpL1GNRhBQM8XgGX3QlBU2nOEVERLyEtZZxkeNoWKYh8UPjHSlnACEFQ5jaeyr7E/fzyA+PkJsP5uRWKmgiIiIOm71lNh0mdqDthLasi1/HwMYDMcY4mqlZhWa81fEtZm2ZxcR1Ex3NkhepoImIiDhs1G+jiIqLwmVd3F7rdvrW6+t0JACebv40LSq0YOiCoRw5fcTpOHmKCpqIiIiDEpMT+X3/7zzW+DGWPbyM2ffOpkiBIk7HAsDP+PHxLR9z+PRhpkRPcTpOnqKCJiIi4qBf9/xKmiuNjtU6Oh0lSw3LNqRycGXCY8KdjpKnqKCJiIg45KdtP/HeqvcIzBdIq0qtnI5zUW2rtGXpnqW4rMvpKHmGVhIQERHxoDRXGp+s/oRV+1adP2149w13E5gv0OFkF9e2SlsmrJvApkObqFuqrtNx8gQVNBEREQ/6YNUHDF0wlKCAIJ5q9hSvt3+dQgGFnI51SW2rtAVg3o55KmgeolOcIiIiHhKdEM2wJcO4reZtJL6UyPtd3ycof5Djt9S4nCrFqtC6UmveX/U+Z9POOh0nT1BBExERcaOnfn6K0LdDeXj2w3T+ujPFAosxpvsYry9lfzX85uHsT9zPLd/cwgerPlBRczMVNBERETdJSEpgXOQ4ggsEM3f7XEIKhrDggQWUL1re6WhXrH3V9jzV7CliT8by9C9Pc+vkW52O5NM0B01ERMRNxkaMJTk9mbl951K7RG2n41wTYwzvd32f97u+z0sLX2LkipEcOX2E0EKhTkfzSTqCJiIi4gYu6+LzNZ/TuXrnXF/O/qp7ze7AuXu4iXuooImIiOSQE2dPnH+8bM8yYk/G8lD9h5wL5CZNyjWhYL6CLN2z1OkoPksFTUREJAd8t/E7Qt4KYeTykczdPpdRv42icP7C9Kjdw+loOa5AvgK0rNhSqwu4keagiYiIXKPjZ48zZN4Q8vvn56VFL50f79+wv9ff4+xqdanehX8u/CcrY1fSomILp+P4HBU0ERGRa/T6r68Tfyqe3/r/xpq4NdQMrUnFohWpFlLN6Whu80STJ3j/9/d5+penWdl/JX5GJ+VykgqaiIjINYg9EcvoP0bzYP0HaV6hOc0rNHc6kkcUzl+Yf7f7N/1/6M/i3Yu9drH33Ep1V0RE5Bp89MdHuKyLEW1HOB3F4/rW60uJQiUYEzHG6Sg+RwVNRETkGszfNZ9WlVpRuVhlp6N4XIF8BXi4wcPM3jKbvSf2Oh3Hp6igiYiIXKVDSYdYe3AtHavm3dN7g5oMIsA/gGd+ecbpKD5FBU1EROQqLYlZAkCHah0cTuKcysUqM/zm4czcPJMao2swd/tcpyP5BLcVNGPMF8aYBGNMdKaxb40xazN+YowxazPGqxhjzmTaNtZduURERK5VmiuNXt/2os+MPhQtUJSwcmFOR3LUsy2e5Y32b5CUksSo30Y5HccnuPMqzq+Aj4CJfw5Ya+/587Ex5h3gRKb9d1prG7gxj4iISI54edHLzNoyi0cbPUr3Gt3J55e3b4oQ4B/ASze9RKorlVfDXyX2RCwVgys6HStXc9sRNGvtr8DRrLYZYwxwNzDFXZ8vIiLiDsfPHuf9Ve/zUIOH+PS2T31ypYCr1bdeXwAmb5jscJLcz6k5aDcB8dba7ZnGqhpjoowxS40xNzmUS0RE5JJ+2vYTqa5UBjYa6HQUr1O9eHVuqnQTYyPHku5KdzpOruZUQevDhUfP4oBK1tqGwLPAZGNM0axeaIwZaIyJMMZEHDp0yANRRUREzklOS2b65umUK1KOZhWaOR3HKz3d/Glijscwe+tsp6Pkah4vaMaYfMAdwLd/jllrk621RzIeRwI7gZpZvd5a+6m1NsxaG1ayZElPRBYRkTxu97Hd3PTlTRR6oxCztsyiV+1eWtroInrU6kHVYlV5NfxVTqeedjpOruXErMaOwBZr7b4/B4wxJYGj1tp0Y0w1oAawy4FsIiKSByUmJzJsyTASkhL+tq1s4bKs2r+K6IRoXmj1AiULlaRPvT4OpMwd/P38+aT7J9zyzS08+dOTfNXzK6cj5UpuK2jGmClAW6CEMWYf8Kq1djxwL3+/OKAN8JoxJg1IBx631mZ5gYGIiEhOe2HhC4yNGMt1xa/727aY4zGkulL5qsdX9GvQz4F0uU/X67ry8k0v8/qy1xnQaACtK7V2OlKuY6y1Tme4amFhYTYiIsLpGCIikotFHogk7LMwnmn+DO92efdv2/cc38OauDX0rN2TczchkOxISkmi5kc1KV+kPKsGrNIpYcAYE2mtzdZN8/S/loiI5Gk/bP0BP+PHsDbDstxeuVhletXppXJ2hYLyBzGyw0hWH1jNx3987HScXEcFTURE8rTFMYtpXLYxIQVDnI7ic+6/8X66XdeNFxe9SFxinNNxchUVNBERybNOpZxi1b5VdKiad9fSdCdjDKO7jeZs2lk+/P1Dp+PkKipoIiLik37Z8Qu1P6pNtQ+qUefjOmxM2Pi3fZbvXU6aK432Vds7kDBvqF68OnfUuYMxEWOYtnEaKekpTkfKFVTQRETE55xKOUX/H/qT6kqldaXW7Dq2i/FR4/+230/bfqJgvoK0qtTKgZR5x8utX8ZlXdwz/R76/9Cf3HyBoqeooImIiE8Z8vMQir9VnP2J+/m619dM7DWRLtW7MH3T9AuKgcu6+H7L93S5rguFAgo5mNj3NSzbkITnExjWZhiT1k/i243fXv5FeZwKmoiI+IyDpw4yNmIsrSq1Ytqd02hZsSUAd15/J7EnY3l50css2rWIo2eOEh4Tzv7E/dxR+w6HU+cNgfkCGd52OOWLlOeHrT84HcfrObGSgIiIiFt8FvkZqa5Uxt06jpqh/1sx8PZatxNaMJSRK0YycsXI8+P5/PJxa81bnYiaJ/kZP5qWb0rEAd3D9HJU0ERExGd8s+EbOlTtcEE5AygWWIy45+I4lXKKJTFLiD0RC0CtErV0ew0Pa1KuCd9v+Z7jZ49TLLCY03G8lgqaiIj4hJPJJ9l6ZCv333h/ltsD/AMIKRjCHXV0StNJYeXO3Ug/8kAkHarp9iYXozloIiKSq7msi1eWvML3m78HoFHZRg4nkktpXK4xABEHIrDWkpqeesH25LRkRoSPYPX+1U7E8xo6giYiIrnaxoSN/PvXfxMUEASooHm74gWLUyu0FjO3zGT70e3M3zmfpQ8tpWpIVQ4kHuD+mfezJGYJX6//mugnownMF+h0ZEfoCJqIiORqUQejAEhKTaJM4TKUKVzG4URyOc+1eI4/9v/B+KjxxJ6Mpds33YhOiOaGT25g5b6VDGk6hJ3HdvLw7IeZu31uljcZ9nU6giYiIrlaVFzU+cc6epY7PNTgId7+7W3OpJ7h/a7vc9d3d9Hp606kpKew9rG11CpRi4IBBXl35btMjZ4KwE2VbqJ2idr8t/N/KVqgqMPfwP1U0EREJFeLOhhFWLkwDIbba97udBzJhgD/AJY9vAxrLWUKl6F1pdYs37ucoS2GUqtELQBGdhzJcy2eY+exnSzctZDZW2fz2ZrPCAkM4a1Obzn8DdzP5OblFsLCwmxEhO6lIiKSV7msi5C3Quhbry+fdP/E6ThylVbGruSZX55h9r2zKV249EX3e2jWQ0yJnkL0E9HUCK2RY5//07afKF+0PA3KNMix98yKMSbSWhuWnX01B01ERHKt3cd2czL5JA3LNHQ6ilyDFhVbsGrAqkuWM4A3OrxBwXwFeWj2QxxKOpQjn3349GH6zerHM788kyPvl1NU0EREJNf6Y/8fwP/urSW+rVyRcnx8y8f8Fvsbpf5bipcWvnRN7+eyLp755RlOJJ9gdLfROZQyZ6igiYhIrvVb7G8EBQRRr3Q9p6OIh9xX7z7m9JlDj1o9GPXbqKu+wjPNlUafGX2YtH4SL7d+mbql6uZw0mujgiYiIrnWyn0raVq+Kfn8dM1bXmGMoXvN7nx222cUKVCEZ+c/y9XMp3950ctM2ziNkR1GMrzt8JwPeo1U0EREJFc6nXqadfHraFGhhdNRxAElg0ry6s2vMn/nfH7a/tPftu85vofktOQLxhKTE5m8YTKPz3mcUb+N4omwJ3ih9QsYYzwVO9tU0EREJNeZtH4SoW+HkuZKo0VFFbS8alCTQdQuUZt/LvgnLus6P77r2C5qjK5Bg3ENGLZ4GCtjVwLw2tLX6DuzL+Mix/Fs82f5oOsHTkW/LBU0ERHJdT5f8znBBYK5+4a7aVulrdNxxCEB/gGMaDuCzYc3M2PTjPPjYyPG4rIuUtJT+M+y/9B+Ynvmbp/LZ2s+o1ftXhx87iDvdHmHAP8AB9NfmgqaiIjkKglJCSzbu4yBjQfy7Z3fUjh/YacjiYN61+lNrdBajFg6gn0n9zEifASfr/mcnrV7snPIThKGJlApuBLdJ3fnRPIJnm/5/GVv5+ENNKtSRERylRmbZuCyLnrX6e10FPEC/n7+jOo0itun3k7N0TU5m3aWIgWK8FyL54Bzc9VW9l/JR398xKmUUzSv0NzhxNmjlQRERCTXWLRrEbdNuY3aJWoTOTDSKyd3izNeXfIq7616j9n3zqZd1XZOx8nSlawkoIImIiK5RsNxDUlKSWL5I8spFVTK6TjiZVLTU716XpmWehIREZ+TkJTA2oNr6Ve/n8qZZMmby9mVUkETEZFcYeGuhQB0rt7Z4SQi7qeCJiIiucKCXQsICQyhUdlGTkcRcTsVNBERyRWWxiylXdV2+Pv5Ox1FxO1U0ERExOudOHuC3cd3E1Y2W/OrRXI9FTQREfF66+PXA1C/TH2Hk4h4hgqaiIh4vXXx6wCoX1oFTfIGFTQREfF6aw+uJbRgKOWKlHM6iohHqKCJiIjXSklPYcjPQ/hl5y80KNNAKwdInqGCJiIiXmvOtjmM/mM0+07uo0m5Jk7HEfEYLZYuIiJe68u1X1K2cFnm3DeHmqE1nY4j4jE6giYiIl5h1IpR3PXdXayMXUlqeirjIsbx8/afebD+gzQq24jC+Qs7HVHEY3QETUREHHcm9Qyv/foaSSlJzNk2hx61evDtxm+5oeQNDGoyyOl4Ih7ntiNoxpgvjDEJxpjoTGPDjTH7jTFrM35uybTtJWPMDmPMVmNMF3flEhER7zNn2xxOpZxiYq+JFPAvwLcbv+Wxxo+x4YkNVAyu6HQ8EY9z5xG0r4CPgIl/GX/PWvvfzAPGmOuBe4EbgHLAQmNMTWttuhvziYiIl5gcPZmyhcvSp24fAvMF8vmazxnVaZSu2pQ8y21H0Ky1vwJHs7l7D2CqtTbZWrsb2AE0dVc2ERHxHtuObOOHrT9w/4334+/nz53X38m8++dRpEARp6OJOMaJiwQGG2PWZ5wCDckYKw/EZtpnX8aYiIj4uBFLRxCYL5ChLYc6HUXEa3i6oI0BqgMNgDjgnYzxrI5h26zewBgz0BgTYYyJOHTokHtSioiIR8QlxjE1eipPhj1JqaBSTscR8RoeLWjW2nhrbbq11gV8xv9OY+4DMs8CrQAcuMh7fGqtDbPWhpUsWdK9gUVExK2+3fgtLuuif6P+TkcR8SoeLWjGmLKZnvYC/rzC8wfgXmNMAWNMVaAG8Icns4mIiGfFn4pn0vpJNCjTgNolajsdR8SruPM2G1OAlUAtY8w+Y0x/4G1jzAZjzHqgHfAMgLV2IzAN2ATMAwbpCk4RkdzJWnv+z2GLh7EyduXf9vlp20+Uf7c8kXGR9K3X19MRRbye+fM/pNwoLCzMRkREOB1DREQy/Lz9Zx6c9SBrBq4hOT2ZGqNrUCywGCv7r6R2idpsTNjIxHUTmbZpGgXzFeT19q/TrUY3AvMFOh1dxO2MMZHW2rDs7KuVBEREJMfM2zGPw6cP887Kd6hbqi4A6a50Hpn9CCseWcHbv73NxHUTMRiW9FvCzVVudjixiHfSWpwiIpJjIuMiAfhszWdM2ziNMoXL8N/O/2XlvpXM3jqbhbsW0rN2T7b9Y5vKmcglqKCJiEiOSHelE3Uwis7VO5PuSmfBrgW0rdKWRxo+Qq3QWjw25zEOJB6ge43uXFf8Oqfjing1FTQREckRW49s5XTqae6rex9jbx0LQMeqHcnnl4/R3UaTkJQAQKdqnZyMKZIraA6aiIhcszOpZ5i+aToAjcs1pm6pujQo0+D8PLRO1TvxUIOHiE6IpnKxyk5GFckVVNBEROSaPf7T40xcN5GShUqev6dZgzINLthn/O3jnYgmkivpFKeIiFwTl3Xx8/af6VGrB9v+sY18fln/7u9n/PAz+mdHJDv0X4qIiFyT6IRoDp0+RM/aPSkWWMzpOCI+QQVNRESuyaJdiwDoULWDw0lEfIcKmoiIXJPFMYupUbwGFYMrOh1FxGeooImIyDVZvX81LSu2dDqGiE9RQRMRkasWlxhHfFI8Dcs0dDqKiE9RQRMRkau29uBaABqWVUETyUkqaCIictWiDkYBUL90fYeTiPgWFTQREblqaw+upVpINYIDg52OIuJTVNBEROSqrYlb87cVA0Tk2qmgiYjIVYk9EcvOYztpVbGV01FEfI4KmoiIXJVFu8/doLZjtY4OJxHxPSpoIiJyVRbuWkipoFLUK1XP6SgiPkcFTURErpi1loW7FtKxWkeMMU7HEfE5KmgiInLFohOiiU+Kp2NVnd4UcQcVNBERuWILdy0ENP9MxF1U0ERE5Iot3L2QWqG1tEC6iJuooImIyBVJSU9hacxSHT0TcSMVNBERuSJRcVEkpSbRrko7p6OI+CwVNBERuSJr4tYA0KR8E4eTiPguFTQREbkia+LWEFowlIpFNf9MxF1U0ERE5IpExkXSqGwj3f9MxI1U0EREJNuS05KJToimUdlGTkcR8WkqaCIikm3RCdGkulJV0ETcTAVNRESybfbW2fgZP1pXau10FBGfpoImIiLZ4rIuvl7/NR2rdaRckXJOxxHxaSpoIiKSLcv3LifmeAz96vdzOoqIz1NBExGRbPlx64/k989Pj1o9nI4i4vNU0EREJFsWxyymRYUWBOUPcjqKiM9TQRMRkcs6euYoUXFRtK/a3ukoInmCCpqIiFzW0pilWKwKmoiHqKCJiMglTd80nUd/fJSQwBCalm/qdByRPCGf0wFERMT7nDh7gv/8+h8STicwaf0kmpRrwme3fUZ+//xORxPJE9xW0IwxXwC3AgnW2roZY6OA24AUYCfwsLX2uDGmCrAZ2Jrx8lXW2sfdlU1ERC7ubNpZwj4LY/ex3RQMKEinap2YcfcMXRwg4kHuPMX5FdD1L2MLgLrW2huBbcBLmbbttNY2yPhRORMRcciiXYvYcXQHk3tP5uSLJ5l3/zyVMxEPc1tBs9b+Chz9y9h8a21axtNVQAV3fb6IiFyd77d8T9ECRelZuyfGGKfjiORJTl4k8Ajwc6bnVY0xUcaYpcaYm5wKJSKSV8UlxtFnRh9mbp5J9xrdNd9MxEGOXCRgjPk/IA34JmMoDqhkrT1ijGkMzDLG3GCtPZnFawcCAwEqVarkqcgiIj7vx20/MjV6KgD31bvP4TQieZvHC5oxph/nLh7oYK21ANbaZCA543GkMWYnUBOI+OvrrbWfAp8ChIWFWU/lFhHxdRviN1A4f2F2P7WbEoVKOB1HJE/z6ClOY0xX4AXgdmvt6UzjJY0x/hmPqwE1gF2ezCYiktdtSNhA3VJ1Vc5EvIDbCpoxZgqwEqhljNlnjOkPfAQUARYYY9YaY8Zm7N4GWG+MWQdMBx631h7N8o1FRCTHWWvZkLCBeqXqOR1FRHDjKU5rbZ8shsdfZN8ZwAx3ZRERkUuLOxXH0TNHVdBEvISWehIRETbEbwCgXmkVNBFvoIImIiKsi18HoCNoIl5CBU1ERIg4EEHVYlUJLRTqdBQRQQVNREQ4V9DCyoU5HUNEMqigiYjkcYdPH2b38d0qaCJeRAVNRCSPizwQCUCTck0cTiIif1JBExHJ4yIOnFu0pVHZRg4nEZE/qaCJiORxqw+splZoLYIDg52OIiIZVNBERPI4XSAg4n1U0ERE8rC4xDj2J+5XQRPxMipoIiJ52J/zz3SBgIh3UUETEcnDIg5E4Gf8aFCmgdNRRCQTFTQRkTzqdOpppkRPoUGZBgTlD3I6johkks/pACIi4oyh84ey/eh2Fj+42OkoIvIX2SpoxpiSwKNAlcyvsdY+4p5YIiLiTl+t/YoxEWN4vuXztKvazuk4IvIX2T2CNhtYBiwE0t0XR0REckJyWjL5/fNjjPnbtuNnjzN47mDaV23PGx3ecCCdiFxOdgtaIWvtC25NIiIiOcJlXbT6ohUu6+Kn+36ibJGyAMSfiqd04dJMWDuBpNQkRnUaRT4/zXQR8UbZvUhgjjHmFrcmERGRHDF3+1wi4yJZF7+Ou767C4D5O+dT7t1yrIlbwycRn9C8QnMt7STixbJb0J7iXEk7a4xJzPg56c5gIiJydUb9NoqKRSvy6s2vsiJ2BftP7mdpzFJc1sXjcx5n25FtDGoyyOmYInIJ2Spo1toi1lo/a21gxuMi1tqi7g4nIiKXt/nQZnpP682JsydYsnsJv+75lWdbPEvvOr0B+HHbj0TGRQLn1t0sWagkd11/l5ORReQysj35wBhzO9Am42m4tXaOeyKJiMiV+OfCfzJn2xy6VO/ChHUTKF+kPI+HPU4B/wJUD6nO7K2ziYyLpEShEhw+fZgBjQZQIF8Bp2OLyCVk9zYbI4EmwDcZQ08ZY1pba190WzIREbmspTFLmbPt3O/L/7f4/zh8+jBju48lMF8gAL3r9GbUb6OwWN7t/C7Hzx5nSLMhTkYWkWzI7hy0W4BO1tovrLVfAF0zxkRExCFzt8+l6zddqVKsCkNbDOXw6cM0KNOAAY0GnN/nn63+SUjBEABaVmzJiHYjCC0U6lRkEcmmK1nqqVimx8E5HURERLLPWsvQ+UOpWqwqvw/4ncFNB1O7RG3GdB+Dv5//+f1CC4Xyfpf3ub7k9dQvU9/BxCJyJbI7B+1NIMoYswQwnJuL9pLbUomIyCUtiVnC5sN6gy8SAAAgAElEQVSb+arHV5QKKgXA5kGbs9z3gfoP8ED9BzwZT0SuUbYKmrV2ijEmnHPz0AzwgrX2oDuDiYhI1hKSEnhh4QuEFgzlnrr3OB1HRNzgkgXNGFPbWrvFGPPn3Qz3ZfxZzhhTzlq7xr3xREQkM2stXSd1ZfPhzUy+Y/L5iwFExLdc7gjas8BA4J0stlmgfY4nEhGRiwqPCSfqYBSf3/Y5ver0cjqOiLjJJQuatXZgxsNu1tqzmbcZY/Rrm4iIh320+iNCC4ZyX737nI4iIm6U3as4f8vmmIiIuElCUgKzt8zmkYaPUDCgoNNxRMSNLjcHrQxQHihojGnIuQsEAIoChdycTUREMpm2cRrpNp0H6z/odBQRcbPLzUHrAjwEVADezTSeCLzspkwiIpKFyRsmU69UPeqWqut0FBFxs8vNQZsATDDG9LbWzvBQJhER+Ytdx3axct9K3uzwptNRRMQDsnsftBnGmO7ADUBgpvHX3BVMRET+Z2r0VADurXuvw0lExBOydZGAMWYscA/wD87NQ7sLqOzGXCIiksFayzcbvqFVxVZUKVbF6Tgi4gHZvYqzpbX2QeCYtXYE0AKo6L5YIiLyp8i4SDYd2qRba4jkIdktaH/eA+20MaYckApUdU8kERHJbMTSEYQEhtC3Xl+no4iIh2S3oP1ojCkGjALWADHAFHeFEhERiEuM4x9z/8GcbXMY2nIowYHBTkcSEQ+57EUCxhg/YJG19jgwwxgzBwi01p5wezoRkTzKWkvfmX1ZEbuCXrV7MaTZEKcjiYgHXfYImrXWRaa1OK21ySpnIiLuNW3jNJbELOH9Lu8z856ZFM5f2OlIIuJB2T3FOd8Y09sYYy6/6/8YY74wxiQYY6IzjRU3xiwwxmzP+DMkY9wYYz40xuwwxqw3xjS6ks8SEfElE9ZNoHpIdQY2Hnj5nUXE52S3oD0LfAckG2NOGmMSjTEns/G6r4Cufxl7kXOnTGsAizKeA3QDamT8DATGZDObiIhPcVkXK/etpH3V9vj7+TsdR0QckK2CZq0tYq31s9bmt9YWzXheNBuv+xU4+pfhHsCEjMcTgJ6Zxifac1YBxYwxZbP3NUREfMemQ5s4fvY4rSq2cjqKiDgkWysJGGPaZDWeUcCuVGlrbVzG6+OMMaUyxssDsZn225cxFncVnyEikmut2LsCgFaVVNBE8qpsFTTg+UyPA4GmQCTQPgezZDW/zf5tJ2MGcu4UKJUqVcrBjxcR8Q4rYldQKqgU1UOqOx1FRByS3bU4b8v83BhTEXj7Kj8z3hhTNuPoWVkgIWN8HxeuTlABOJBFlk+BTwHCwsL+VuBERHK7FbEraFWxFVd4XZaI+JDsXiTwV/uAulf52h+AfhmP+wGzM40/mHE1Z3PgxJ+nQkVE8oqDpw6y69guzT8TyeOyOwdtNP873egHNADWZeN1U4C2QAljzD7gVWAkMM0Y0x/Yy7mF1wHmArcAO4DTwMPZ/hYiIj5C889EBLI/By0i0+M0YIq1dsXlXmSt7XORTR2y2NcCg7KZR0TEJ62IXUFgvkAaldWtIEXysuzOQZtgjCmZ8fiQeyOJiORN1lrCY8JpUq4J+f3zOx1HRBx0yTloGfPBhhtjDgNbgG3GmEPGmFc8E09EJO+Yu30uUQejuPuGu52OIiIOu9xFAk8DrYAm1tpQa20I0AxoZYx5xu3pRETyCGst/1z4T2qG1uSxxo85HUdEHHa5gvYg0Mdau/vPAWvtLuD+jG0iIpIDNh/ezKZDm3iuxXME+Ac4HUdEHHa5ghZgrT3818GMeWj6G0REJIcs3LUQgM7VOzucRES8weUKWspVbhMRkSuwaPciqodUp0qxKk5HEREvcLmrOOsbY05mMW44t+STiIhcozRXGuEx4dx7w71ORxERL3HJgmat9fdUEBGRvGrdwXWcTD5Ju6rtnI4iIl7iapd6EhGRHLJq3yoAWlZs6XASEfEWKmgiIg5btX8VZQuXpWLRik5HEREvoYImIuKwVftW0bxCc4wxTkcRES+hgiYi4qDDpw+z4+gOmldo7nQUEfEiKmgiIg6asWkGAK0rtXY4iYh4ExU0ERGHpKSn8MbyN2hWvhktKrRwOo6IeJHL3QdNRETc4GTySR6Z/Qh7T+xl3K3jNP9MRC6gI2giIg4YtWIU32/5nlGdRtGlehen44iIl9ERNBERB6zct5IGZRowtOVQp6OIiBfSETQREQ+z1hJxIIIm5Zo4HUVEvJQKmoiIh+08tpMTyScIKxfmdBQR8VIqaCIiHhZxIAJABU1ELkoFTUTEw1bvX01gvkBuKHmD01FExEupoImIeNjy2OWElQsjwD/A6Sgi4qVU0EREPCgxOZHIA5G0rdzW6Sgi4sVU0EREcsjSmKWs2rfqottd1sWK2BWk23RurnKzB5OJSG6j+6CJiOSA2BOxdJ/cnZJBJdk5ZCd+5sLff13WRdinYWw/up0AvwAt7SQil6QjaCIi18hay5B5Q0hKTSLmeAwr9q742z7L9iwj6mAUZ1LP0KZyG4LyBzmQVERyCxU0EZFr9PHqj5m1ZRbDbx5OUEAQX679EoDktGS+jf6Ws2ln+XLtlxTJX4Q9T+9h+t3THU4sIt5OpzhFRK7Bol2LePaXZ7m15q0Mu3kYB08dZGzkWIoWKEqZwmV4adFLdKzWkeV7l3N/vfspX7S805FFJBdQQRMRuUrxp+K5Y9od1Aytyde9vsbP+PFBtw/I55ePD37/AH/jT2jBUBbuWkjdUnV5rd1rTkcWkVxCBU1E5CpNWj+Jk8knmXbXNIoFFgMgv39+Puz2IceTj/PN+m+Yc98c4k/F07ZKW4IDgx1OLCK5hbHWOp3hqoWFhdmIiAinY4hIHlV/bH0K5ivIqgF/v7VGuiudmOMxVC9e3YFkIuKNjDGR1tpsrfGmiwRERK7C+vj1rI9fz4P1H8xyu7+fv8qZiFw1FTQRkauwePdiAHrU6uFwEhHxRSpoIiJX4bfY36gcXFlXZYqIW6igiYhchZX7VtKiolYDEBH3UEETEblCsSdi2XdyHy0rtHQ6ioj4KBU0EZErtCRmCQAtK6qgiYh7qKCJiFyBUymnGLZkGNeXvJ76Zeo7HUdEfJRuVCsikk3WWp76+Sn2ntjLikdWkM9Pf4WKiHt4/G8XY0wt4NtMQ9WAV4BiwKPAoYzxl621cz0cT0Tkbw6eOsjag2uZsWkGX6z9gv+76f90elNE3MrjBc1auxVoAGCM8Qf2A98DDwPvWWv/6+lMIiIXsyZuDW2/aktiSiIBfgE83vhxrakpIm7n9PH5DsBOa+0eY4zDUURELnQm9QzdJ3cnpGAIs++dTcOyDc+vuSki4k5OXyRwLzAl0/PBxpj1xpgvjDEhToUSEQFYtHsRB08dZNyt42hXtZ3KmYh4jGMFzRiTH7gd+C5jaAxQnXOnP+OAdy7yuoHGmAhjTMShQ4ey2kVEJEf8sPUHiuQvQvuq7Z2OIiJ5jJNH0LoBa6y18QDW2nhrbbq11gV8BjTN6kXW2k+ttWHW2rCSJUt6MK6I5CWnU0/z47Yf6VajG/n98zsdR0TyGCcLWh8ynd40xpTNtK0XEO3xRCIiQFRcFKX/W5qDpw5y9/V3Ox1HRPIgRy4SMMYUAjoBj2UaftsY0wCwQMxftomIeMzoP0ZjMCx8YCEdqnVwOo6I5EGOFDRr7Wkg9C9jDziRRUTkT6npqew9sZfvNn3H3dffrXImIo5x+ipOERGvMTx8ONeNvo5TKafo16Cf03FEJA9z+j5oIiJe46ftP1GnRB0GNBrATZVucjqOiORhOoImIgIcOX2EdfHr6FO3D8+2eBbdPFtEnKQjaCKS58Ucj2H+zvkAtKvazuE0IiIqaCKSh1lruWPaHczaMguAQgGFaFo+y1swioh4lAqaiORZkXGRzNoyiyfDnsRiKV+kvG5KKyJeQQVNRPKEz9d8zpiIMYSVDaNkUEn+1eZffLP+G/L75+f1Dq9rnU0R8SoqaCLi886mneVfi/+Fy7qIOR7D0TNHKV6wOFOip3BrzVtVzkTE66igiYjPclkXvaf1ZsvhLcQnxbPwgYW0r9qell+05Ln5zwEwuMlgh1OKiPydbrMhIj7HWsvkDZN5aeFLzNoyi4OnDtKyYkvaV22PMYaXWr8EwHMtntNVmyLilXQETUR8zqLdi+g7sy8A7aq0Y/4D87HWnr+32e21bmfNwDXcWPpGJ2OKiFyUCpqI+Jw3l79JuSLl+KrHVzQq24h8fn//q65h2YYOJBMRyR4VNBHxKX/s/4PFuxfz307/pVP1Tk7HERG5KpqDJiI+5c3lbxISGMLAxgOdjiIictV0BE1Ecg2XdbE+fj03lr4RP/O/3y/DY8IJjwmnWflmzNoyi2FthlGkQBEHk4qIXBsVNBHxWsfPHiclPYVSQaWIPRFLj6k9iDoYxQM3PsD428cT4B9AYnIi9824j7hTcQBUC6nGU82ecji5iMi1UUETEUfM2DQDP+NHrzq9zo8lpSTx7sp36VOvD6WDStPs82bsPLqTgY0Hku5KZ9OhTfRv2J/xUeM5kXyCO2rfwdSNU4k7FccHXT/gUNIhhrYcSnBgsIPfTETk2qmgiYjHnUk9w4AfBxAUEETP2j0xxnA69TTdvunGsr3LGBMxhurFq7Pj6A561e7FmIgxGAyPNHyEz2//nIZlGjL458H8sPUHQgJDGH7zcIY0G+L01xIRyTEqaCLicdM3Tef42eMcP3ucqINRVAquxKT1k1i2dxlvtH+DT9d8yu5ju/n4lo8Z2Hggbb9qy7K9y86fuhzUdBBtKrcB4IZSN1wwH01ExBcYa63TGa5aWFiYjYiIcDqGiFwBay0tv2jJvpP72HdyH6EFQ0lMSSS4QDB1StZh6UNLL7ipLMDh04dZe3AtHat1dDC5iMi1McZEWmvDsrOvfu0UEY+avmk6q/at4v9u+j/CyoVx5MwRggsEc+j0IYa2GApwQTkDKFGohMqZiOQpOsUpIh7jsi6GLhhKgzINeLTRo1QoWoHIA5EMbjqY8Jhwbq15q9MRRUS8ggqaiGRLVFwUtUrUYt6OeUyJnkK5wuUY3nY4IQVDsv0eu47tYu+JvQxrMwx/P39urXnr+VLW+/re7oouIpLrqKCJyGUt27OMNl+1oVn5ZkTGRVI6qDQHTx1k1tZZLH94ORWDK2brfTbEbwDQIuUiIpehOWgickmp6ak8OfdJggKC+H3/74QWDGX9E+v5rf9vnDh7gnYT2jF+zXhc1nXZ91ofvx6D4YaSN3gguYhI7qUjaCJySa8ve53ohGhm3TOL2JOxNC7bmOIFi9O0fFPm3DeHgT8OZMCPA/Azfjzc8OFLvteGhA1cV/w6gvIHeSi9iEjupIImIhe19uBa/vPrf7j/xvvpUbvH37a3rtSajU9upOnnTXnt19e48/o7sVie/OlJTqWcYkCjAdxc+ebz62Kuj19PvdL1PP01RERyHRU0EQEgISmB/j/0p2GZhvSt15cVsSuYsXkGRQsU5cOuH170dcYYXm//Ol0mdSF4ZDD+fv5YaykWWIzZW2dTolAJlj+8nKD8Qew4uoO+9fp68FuJiOROKmgiwrwd83j0x0fZd3IfC3ct5IetP7Aufh0Ab7R/47JXanau3pnwfuEs3bOU06mn6VGrBw3LNmRpzFIe+P4Bmo9vTnJaMgH+AdxS4xZPfCURkVxNKwmI5HGLdy+m48SO1ClZhxdavUC/Wf0A6Fm7JynpKUztPfX8KcqrsXr/at5a8RblipRjSLMhXFf8upyKLiKSq1zJSgIqaCJ52JLdS+g7sy9FCxQlcmAkQfmDaPtVW3Yc3cGOITsIzBfodEQREZ9xJQVNpzhF8qCVsSv515J/sXj3YsoXKc+0u6adv7Jy+t3TOZN6RuVMRMRBKmgiecjmQ5sZumAoc7fPpVRQKd7v8j6PhT12QRkrUaiEgwlFRARU0ETyjKNnjtLp606cTj3Nmx3eZHDTwRTOX9jpWCIikgUVNBEfFRUXxTsr3+HomaMMaDSAL6K+ICEpgVUDVtGobCOn44mIyCWooIn4mDVxa+j/Q3/WHlxLSGAIhfMXpve0cwuRj+k+RuVMRCQXUEET8QEjl48kuEAwTzR5gleWvELsiVjeaP8GTzR5gvz++RkePpxWFVtluRqAiIh4HxU0kVzu2JljDFsyjDRXGjuP7eTnHT/zQqsXeOmml87v83antx1MKCIiV0oFTSSXm7t9LmmuNFpXas07K98B4OEGl160XEREvJsKmkguMTV6KlsOb2F42+EXjM/aOouyhcuy9KGl5y8EqBFaw5mQIiKSIxwraMaYGCARSAfSrLVhxpjiwLdAFSAGuNtae8ypjCKekuZKw8/44Wf8stw+Y9MM+s7si8u6aFulLfVL12fZ3mUUzl+YOdvm0K9+P/yMHwMaDfBwchERcQenj6C1s9YezvT8RWCRtXakMebFjOcvOBNNxP2SUpLo9W0vFuxaQMMyDVn96Gr8/fzPbz+bdpYXFrzAh398SNPyTTmQeIB7pt9DYnIiZ9LOAFApuBKv3PyKU19BRETcIOtf153TA5iQ8XgC0NPBLCJuFXsilq7fdGXR7kXcfcPdRB2MYubmmaS70gFISU+h9Ret+fCPD/lH038Q3i+cdzu/S+mg0vRv2J85febwfMvnmX//fMoVKefwtxERkZzk2GLpxpjdwDHAAuOstZ8aY45ba4tl2ueYtTbkYu+hxdIlt5q0fhKD5w4mzZXG+NvHc+f1d1Ln4zrsObEHP+PHO53fIcAvgIFzBvJ1r6+5/8b7nY4sIiLXKLcslt7KWnvAGFMKWGCM2ZKdFxljBgIDASpVquTOfCI5zlrLiKUjGLF0BK0qtmJCzwlUL14dgHc6v8N7q97DGMOguYPwN/40KdeEvvX6OpxaREQ8zbEjaBeEMGY4cAp4FGhrrY0zxpQFwq21tS72Oh1Bk9wkOS2ZV5a8wtu/vc1DDR7i89s+v2C+2Z9c1sWXUV8yLnIc73R+h5sq3+RAWhERyWlXcgTNkYJmjAkC/Ky1iRmPFwCvAR2AI5kuEihurf3nxd5HBU1yizGrxzBo7iAslifCnuCjWz666BWbIiLim3LDKc7SwPfGmD8zTLbWzjPGrAamGWP6A3uBuxzKJ5JjrLV8+MeH3FDqBobfPJw76txBxv/3RUREsuRIQbPW7gLqZzF+hHNH0UR8xpq4NWw5vIVxt46j9/W9nY4jIiK5gM6xiLjR3O1zGfDjAPL75+eu63VAWEREskcFTcRNNiZs5I5v7+DE2ROM6T6GkIIXvWOMiIjIBZxeSUDEJ7msi36z+lG0QFFW9l9J6cKlnY4kIiK5iAqaiBt8s/4bIuMimdRrksqZiIhcMZ3iFMlh8afieXnxyzQu25g+9fo4HUdERHIhHUETySFHTh/htaWv8dP2nzh65igz756pe52JiMhVUUETuUbRCdGsjF1J+J5wvo3+lvpl6vPZbZ/RpHwTp6OJiEgupYImcpVOJp9k2sZpPPPLM5xKOQXAv276F/9u/2+Hk4mISG6ngiZyFay1dJjYgYgDETQs05AhzYbw+77fefmml52OJiIiPkAFTeQqLN69mIgDEbzX5T2GNBuCn/HjoQYPOR1LRER8hGYwi1yh1PRU3lz+JqWCSvF42OO6EEBERHKcjqCJXIE0VxqdJ3UmPCacD7p+QGC+QKcjiYiID1JBE8mmpJQkwmPCCY8JZ3S30QxuOtjpSCIi4qNU0EQu4eCpg4THhDNp/SR+3vEz1UKqUSqoFI81fszpaCIi4sNU0EQuYtjiYby+7HUsluACwdQpUYeNhzYytMVQAvwDnI4nIiI+TAVNJAuxJ2J5a8Vb3FbrNl5p8wo3lr6RxJRE3l7xNs80f8bpeCIi4uNU0ESy8MayN7BYPuz6IZWLVQageMHijOw40uFkIiKSF6igiWSy8+hOpkZPZWzkWAY1GXS+nImIiHiSCprkeWmuNJbGLGV81HimRE8B4Laat/Fel/ccTiYiInmVCprkWSeTTzJqxSjGRIzhyJkjBAUE8WKrF7m37r3UK11PN6AVERHHqKBJnrL/5H5KFCpBqiuVjhM7svrAanrV7sUDNz5Al+u6UCigkNMRRUREVNAk7wiPCafrpK40Kd+EAL8A1sStYdY9s+hRu4fT0URERC6ggiY+y1rL9E3TmbpxKvfccA8DfhhAyaCSLN+7HH/jz4SeE1TORETEK6mgiU/adWwXg+YOYt6OeRgMMzfPpFJwJVY8soLV+1dTvGBxbq5ys9MxRUREsqSCJj5nwtoJPP7T4+Tzy8f7Xd6nW41ujFoxiudbPU+FohWoULSC0xFFREQuSQVNfMrxs8cZMm8Ijcs2ZuqdU8+Xsc9u/8zhZCIiItmn+wiIT/lk9SecTD7J6G6jdaRMRERyLRU08Rkp6Sl8+PuHdL2uKw3LNnQ6joiIyFVTQZNczVrLrmO7cFkXs7fMJj4pnsFNBjsdS0RE5JpoDprkWtuObKPH1B5sObyFikUr4u/nT6XgSnS9rqvT0URERK6JjqBJrvVq+KvsP7mfUZ1G0aBMA06lnOL5ls/j7+fvdDQREZFroiNokivFHI/hu43f8UzzZxjacihDWw51OpKIiEiO0RE0yXWOnTlGnxl98DN+DGk2xOk4IiIiOU4FTXKV1PRUbp96O2vi1jD1zqlUDK7odCQREZEcp1Ockmu8t/I9pkRPYfWB1UzpPYU76tzhdCQRERG30BE0yRUmrJ3As/Of5UzaGd7p/A731r3X6UgiIiJuoyNo4lUSkxM5mXyS8kXLY61lQ8IGZmyawZvL36RdlXbMf2A++fz0f1sREfFt+pdOvEZcYhztJrRj38l9DGszjO82fUdkXCQAd99wN2O7j1U5ExGRPEH/2onjohOimbJhCmMjx5Kclkz5ouV5cdGL1Aqtxce3fEzX67pSLaSa0zFFREQ8RgVNHOOyLoaHD+c/v/4HYwzda3Tn3+3+TeVilVkfv57WlVrjZzRNUkRE8h6PFzRjTEVgIlAGcAGfWms/MMYMBx4FDmXs+rK1dq6n84nnvLLkFV5f9jr96vfj7U5vUyqo1PltbSq3cTCZiIiIs5w4gpYGPGetXWOMKQJEGmMWZGx7z1r7XwcyiZulpKdwOvU0BkN8UjzTN03n9WWvM6DhAD697VOMMU5HFBER8RoeL2jW2jggLuNxojFmM1De0znEvVLSU9h8aDMbD23k2JljvLH8DQ4kHrhgn/vq3ccn3T9RORMREfkLR+egGWOqAA2B34FWwGBjzINABOeOsh3L4jUDgYEAlSpV8lhWuTxrLVOipzDqt1FsTNhIqiv1/LbrS17PU82ewmVdlC9SnqD8QfSs3VNzzERERLJgrLXOfLAxhYGlwOvW2pnGmNLAYcAC/wbKWmsfudR7hIWF2YiICPeHlWwZuXwkLy16iQZlGtClehfql65PvdL1KBRQiErBlXSLDBERydOMMZHW2rDs7OvIv5jGmABgBvCNtXYmgLU2PtP2z4A5TmSTq7P/5H5eW/oaPWv35Lu7vlMZExERuQZOXMVpgPHAZmvtu5nGy2bMTwPoBUR7OptcmYgDEYz+YzRprjSi4qJIt+m82/ldlTMREZFr5MS/pK2AB4ANxpi1GWMvA32MMQ04d4ozBnjMgWxyGZM3TGZ4+HAC8wWyIWEDwQWCCQ4MplRQKab0nkLVkKpORxQREcn1nLiKczmQ1WV7uueZl0pISmDRrkXM2T6HyRsm06BMA4ILBDOq0ygGNh5I0QJFnY4oIiLiU3QuSrK0/ch20lxpuKyLbt90I/Zk7P+3d/+xVdVnHMffT1tafrTSllJAqEAHUVBRFAhNjTidv9CoOI0aRDJJplEzTUYMGqdgXKL/CJmYqdlksBmZYYuaLVlQUKtDfoOAVRArv5RRgRYoDErbZ3/cL/Vab7ED2nNu+3klN/ec7/nec57DE9Lnfr/nnkOvbr2YUT6DWT+dRXZmdtQhioiIdFoq0OR79tTt4dY3bmXZzmXNbXnZeSy5ZwnjB42nZ7eeEUYnIiLSNahAExZ/uZhnKp6h+nA1fXr2Yc03a3j+mufpn9ufmqM1TBg8gfOLz486TBERkS5DBVoX4u7sOLCDQWcNIsMyOHDsAC+seIGZH8xkaP5QsjKyWLZzGbOvnc0j4x+JOlwREZEuSwVaF+DurN29lmf//SyLKhcxIHcAh48f5uCxgwDcPvJ25t08D4CPdnzE1T+5OspwRUREujwVaJ1Ekzfx8c6PGTtwLNmZ2Wzdv5Wt+7eSl53H3FVzWbhpIdmZ2Uwvm85XtV/Rt2dfhuQP4fLBl1NWUta8n2uHXRvhWYiIiAioQEtLR44f4YmlT1DfWM/kCyezu243H27/kDkr5jCiaASj+o1iUeUiGr2x+TOzrpjFA2MfoKhnUYSRi4iISFuoQGsHh44dYmP1Rgq6F1DfWE9VTRUZlkFZSRnFvYrbvB93p66+jpysHGqP1lLYo5CVX6/k3rfuZfO+zWRYBi+uerG5/6TzJlFVU0XF9gqmjZ7G1IunUnu0lqKeRYwbOK49TlVERETagQq0H9HQ1ND86KJjDceo/LYSgJF9R1J7tJYdB3ZQsb2CjdUbuaD4AvK75zPz/Zl8fejrH+wrKyOL+y69j4fGPUTF9gp2HdxFeUk5y3ctp3JvJYfrD9PQ1MDYs8eSl5PHvPXz+Hzv5z/Yz8C8gbw75V2KehaxqXoTJb1LqKqpYsqoKWRmZLbvP4iIiIi0O3P3qGM4ZWPGjPHVq1e32/6PHD/CuXPPZeKwiUwYMoGnP3iazfs2p+xb3KuY6sPVAFxYfCFPTniS443HycrIorSglPrGehZ8soCX1rzU/BnDcJwMy6C0oJSzcs7C3Vn/n/U4TnlJOROHT6TJm+id05vqw9UMLRjKpPMmUdCjoN3OW6e1CSgAAAb0SURBVERERM48M1vj7mPa0lcjaCdRV1/HNaXXsGDDAl5Z+wrFvYqZf8t8sjOz+WLfFxT2KKR/bn/KSso4O+9sPq3+lIamBkb1G0XimfDfV1ZSxpSLprBl3xZG9x9Nv9x+rPlmDeXnlJPfPb+537babTR5E6UFpR15uiIiIhITGkFrg7r6Orbs28LQ/KEauRIREZFTohG0Myw3O5dLBlwSdRgiIiLSRWREHYCIiIiIfJ8KNBEREZGYUYEmIiIiEjMq0ERERERiRgWaiIiISMyoQBMRERGJGRVoIiIiIjGjAk1EREQkZlSgiYiIiMSMCjQRERGRmFGBJiIiIhIzKtBEREREYkYFmoiIiEjMqEATERERiRkVaCIiIiIxY+4edQynzMy+BbZ3wKGKgL0dcBxpH8pf+lMO059ymP6Uw9M32N37tqVjWhdoHcXMVrv7mKjjkFOj/KU/5TD9KYfpTznsWJriFBEREYkZFWgiIiIiMaMCrW1eiToAOS3KX/pTDtOfcpj+lMMOpGvQRERERGJGI2giIiIiMaMC7STM7Doz22xmW81sRtTxSGpm9qqZVZvZpqS2QjN7x8y+CO8Fod3M7HchpxvM7JLoIhcAMysxs/fM7DMz+9TMHg7tymGaMLPuZrbSzD4JOZwV2oea2YqQw7+aWXZozwnrW8P2IVHGL98xs0wzW2dm/wjrymFEVKC1wswygReB64GRwF1mNjLaqKQVfwKua9E2A1ji7sOBJWEdEvkcHl6/BH7fQTFK6xqAX7v7CGA88GD4v6Ycpo9jwJXufhFwMXCdmY0HngNmhxzWANNC/2lAjbsPA2aHfhIPDwOfJa0rhxFRgda6ccBWd69y93pgIXBzxDFJCu5eAexv0XwzMD8szwduSWpf4AnLgXwzG9AxkUoq7r7b3deG5UMk/jgMRDlMGyEXdWG1W3g5cCWwKLS3zOGJ3C4CrjIz66BwpRVmNgi4AfhDWDeUw8ioQGvdQGBn0vqu0CbpoZ+774ZEAQAUh3blNcbCNMloYAXKYVoJU2PrgWrgHeBLoNbdG0KX5Dw15zBsPwD06diIJYU5wKNAU1jvg3IYGRVorUv1TUA/eU1/ymtMmVku8DfgEXc/eLKuKdqUw4i5e6O7XwwMIjEDMSJVt/CuHMaMmd0IVLv7muTmFF2Vww6iAq11u4CSpPVBwDcRxSL/vz0npr3Ce3VoV15jyMy6kSjOXnP3v4dm5TANuXst8D6J6wnzzSwrbErOU3MOw/be/PAyBelY5cBNZraNxCU9V5IYUVMOI6ICrXWrgOHhFyzZwJ3A2xHHJG33NjA1LE8F3kpqvyf8EnA8cODENJpEI1y38kfgM3d/PmmTcpgmzKyvmeWH5R7Az0hcS/gecFvo1jKHJ3J7G7DUdVPOSLn7Y+4+yN2HkPh7t9TdJ6McRkY3qj0JM5tI4htEJvCqu/824pAkBTN7HbgCKAL2AE8BbwJvAOcAO4Db3X1/KAbmkvjV5xHgF+6+Ooq4JcHMLgM+BDby3bUvj5O4Dk05TANmNorEBeOZJL74v+HuT5tZKYnRmEJgHXC3ux8zs+7An0lcb7gfuNPdq6KJXloysyuA6e5+o3IYHRVoIiIiIjGjKU4RERGRmFGBJiIiIhIzKtBEREREYkYFmoiIiEjMqEATERERiRkVaCLSaZhZo5mtT3rN+JH+95vZPWfguNvMrOh09yMicoJusyEinYaZ1bl7bgTH3QaMcfe9HX1sEemcNIImIp1eGOF6zsxWhtew0D7TzKaH5V+ZWaWZbTCzhaGt0MzeDG3Lww1ZMbM+ZrbYzNaZ2cskPZfQzO4Ox1hvZi+bWWYEpywiaU4Fmoh0Jj1aTHHekbTtoLuPI/EUgjkpPjsDGO3uo4D7Q9ssYF1oexxYENqfAj5y99EkHnlzDoCZjQDuAMrDg8Mbgcln9hRFpCvI+vEuIiJp47+hMErl9aT32Sm2bwBeM7M3STwqDOAy4OcA7r40jJz1Bi4Hbg3t/zSzmtD/KuBSYFXiiVT04LuHvIuItJkKNBHpKryV5RNuIFF43QT8xszOJ2nqMsVnU+3DgPnu/tjpBCoioilOEekq7kh6/zh5g5llACXu/h7wKJAP5AIVhCnK8ADpve5+sEX79UBB2NUS4DYzKw7bCs1scDuek4h0UhpBE5HOpIeZrU9a/5e7n7jVRo6ZrSDxxfSuFp/LBP4Spi8NmO3utWY2E5hnZhuAI8DU0H8W8LqZrQU+AHYAuHulmT0BLA5F33HgQWD7mT5REencdJsNEen0dBsMEUk3muIUERERiRmNoImIiIjEjEbQRERERGJGBZqIiIhIzKhAExEREYkZFWgiIiIiMaMCTURERCRmVKCJiIiIxMz/AN9dNf2BTnBhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1123b3978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.ylabel(\"Duration\")\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.plot(running_mean(time_steps, 50), color='green')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
